
for base_id = 1:length(vali)
    
    frame_n = data_base{ base_id }.frame_n;
    
    validators = {};
    
    vali{ base_id }.act_F_res_mat = zeros(size(vali{ base_id }.mean_score_res_mat));
    vali{ base_id }.act_F_lab_mat = zeros(size(vali{ base_id }.mean_score_lab_mat));
    
    vali{ base_id }.act_F_res = zeros(size(vali{ base_id }.mean_score_res));
    vali{ base_id }.act_F_lab = zeros(size(vali{ base_id }.mean_score_lab));
    
    handle_F = [];
    
	if(base_id==1),continue,end;
    
    for i = 1:frame_n
        
        
        
        frame_id = data_base{ base_id}.start ...
            + data_base{ base_id}.rate *(i-1);
        
        %get ground truth
        
        msk_name = [    msk_root ...
                        data_base{ base_id}.name '/' ...
                        data_base{ base_id}.name ...
                        '_maskgt_' ...
                        num2str(frame_id) '.jpg'];
        
        try
            gt_img = imread(msk_name);
        catch
            continue;
        end
        
        if(size(gt_img,3)==3)
            gt_img = rgb2gray(gt_img);
        end
        gt_img = im2double( gt_img );
        
        
        
        %pick up response                
                
        pick_id = vali{ base_id }.pick_id( i , : );
        best_pick_id = vali{ base_id }.best_pick_id( i , : );
        
        pick_num = length(pick_id);
        
        if(isempty(validators))
            for k = 1:pick_num
                validators{k} = getValidator([0 0 0 0] );
                best_validators{k} = getValidator([0 0 0 0] );
            end                
        end
        
        clear res
        clear best_res
        
            for k = 1:pick_num
        
                res_fname = [ res_root data_base{ base_id }.name ...
                           '_' feature_code '_' ...
                           num2str( i - 1 ) '_' ...
                           'c' num2str( pick_id(k) - 1 ) '.jpg'];
                       
                res{k} = im2double(imread( res_fname));
                       
                best_res_fname = [ res_root data_base{ base_id }.name ...
                           '_' feature_code '_' ...
                           num2str( i - 1 ) '_' ...
                           'c' num2str( best_pick_id(k) - 1 ) '.jpg'];
                       
                best_res{k} = im2double(imread( best_res_fname ));
            end
            
        gt_img = imresize( gt_img, size( best_res{1} ));
        gt_img = min(1,gt_img+0.3); %enhance get more foreground?
        
        sum_img = zeros( size(gt_img) );
        best_sum_img = zeros( size(gt_img) );
            
            for k = 1:pick_num
                sum_img = sum_img + res{k};
                temp_vali = getValidator( sum_img/k , gt_img);
                best_sum_img = best_sum_img + best_res{k};
                best_temp_vali = getValidator( best_sum_img/k , gt_img);
                
                validators{k} = addValidator( validators{k}, temp_vali);
                best_validators{k} = addValidator( best_validators{k}, best_temp_vali);
                
                
                vali{ base_id }.act_F_res_mat(i,k) = temp_vali.F1;
                vali{ base_id }.act_F_lab_mat(i,k) = best_temp_vali.F1;
                
                vali{ base_id }.act_F_res(k) = validators{k}.F1;
                vali{ base_id }.act_F_lab(k) = best_validators{k}.F1;
            end
            
        if(mod(i,10)==0)
            disp([ num2str(i) ' / '  num2str( frame_n ) ] )
            
            disp( [num2str(max(vali{base_id}.act_F_res)) ' ' vali_name]);
            
            if(0)%if(base_id<=visual_id)
            
                figure(4)

                imagesc( [ vali{ base_id }.act_F_res_mat ...
                            vali{ base_id }.act_F_lab_mat] )
                drawnow;
            
                figure(5)
            
            
                handle_F = plot( 1:pick_num, vali{ base_id }.act_F_res, 'r' ,...
                                 1:pick_num, vali{ base_id }.act_F_lab, 'b' ,...
                                 'LineWidth',2);
                grid on
                xlim([.5 pick_num+.5])
                
                xlabel('Num of Top bag in avg' ,'FontSize',14)
                ylabel('F_1 score')
                temp_name = data_base{base_id}.name;
                temp_name(temp_name=='_') = ' ';
                temp_h = legend( ['on ' temp_name],'Ideally');
                set(temp_h,'FontSize',14)
                set(gca,'FontSize',13);
                             
                
            
                drawnow;
            end
        end
                                
        
    end        
end


%%





save(vali_name,'vali');